{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 微调整正则化参数：reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import math\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 将数据分割训练数据与测试数据\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 获取特征，标签\n",
    "y = train['interest_level']\n",
    "X = train.drop(['interest_level'], axis=1)\n",
    "\n",
    "# 由于数据集较大，在此随机采样30%的数据构建训练样本，其余作为测试样本\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=33, test_size=0.7, stratify=y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "# 各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5], 'reg_lambda': [0.01, 0.05, 0.1]}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 正则参数\n",
    "reg_alpha = [1.5]    #经过多次微调，最终确定了1.5最优（中间调优时的参数集合未列出，仅保留最终一次调优参数）\n",
    "reg_lambda = [0.01, 0.05, 0.1]      #经过多次微调，最终确定了0.1最优（中间调优时的参数集合未列出，仅保留最终一次调优参数）\n",
    "\n",
    "param_test5_2 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.60951, std: 0.00406, params: {'reg_alpha': 1.5, 'reg_lambda': 0.01},\n",
       "  mean: -0.60982, std: 0.00414, params: {'reg_alpha': 1.5, 'reg_lambda': 0.05},\n",
       "  mean: -0.60897, std: 0.00461, params: {'reg_alpha': 1.5, 'reg_lambda': 0.1}],\n",
       " {'reg_alpha': 1.5, 'reg_lambda': 0.1},\n",
       " -0.60897300614028971)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 代入前面获得的最优参数\n",
    "xgb5_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=124,  \n",
    "        max_depth=5,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.9,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_2 = GridSearchCV(xgb5_2, param_grid = param_test5_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_2.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_2.grid_scores_, gsearch5_2.best_params_, gsearch5_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 36.21635709,  33.29824452,  28.94238997]),\n",
       " 'mean_score_time': array([ 0.12744517,  0.12233009,  0.11550679]),\n",
       " 'mean_test_score': array([-0.60951036, -0.60981553, -0.60897301]),\n",
       " 'mean_train_score': array([-0.48950474, -0.48980072, -0.4899637 ]),\n",
       " 'param_reg_alpha': masked_array(data = [1.5 1.5 1.5],\n",
       "              mask = [False False False],\n",
       "        fill_value = ?),\n",
       " 'param_reg_lambda': masked_array(data = [0.01 0.05 0.1],\n",
       "              mask = [False False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'reg_alpha': 1.5, 'reg_lambda': 0.01},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 0.05},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 0.1}],\n",
       " 'rank_test_score': array([2, 3, 1], dtype=int32),\n",
       " 'split0_test_score': array([-0.60444849, -0.60419866, -0.60276856]),\n",
       " 'split0_train_score': array([-0.491083  , -0.49073928, -0.49124366]),\n",
       " 'split1_test_score': array([-0.61643182, -0.61643015, -0.6168597 ]),\n",
       " 'split1_train_score': array([-0.4876991 , -0.48909522, -0.48885366]),\n",
       " 'split2_test_score': array([-0.61037376, -0.61042459, -0.60867854]),\n",
       " 'split2_train_score': array([-0.48906845, -0.48932953, -0.48861725]),\n",
       " 'split3_test_score': array([-0.60669115, -0.60693689, -0.60674875]),\n",
       " 'split3_train_score': array([-0.48843274, -0.48938682, -0.48896028]),\n",
       " 'split4_test_score': array([-0.60960502, -0.61108647, -0.60980843]),\n",
       " 'split4_train_score': array([-0.49124041, -0.49045275, -0.49214364]),\n",
       " 'std_fit_time': array([ 1.2952219 ,  0.33257479,  3.21029926]),\n",
       " 'std_score_time': array([ 0.00543151,  0.00513443,  0.01460979]),\n",
       " 'std_test_score': array([ 0.00405595,  0.00413757,  0.00461431]),\n",
       " 'std_train_score': array([ 0.0014215 ,  0.00066289,  0.00144516])}"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_2.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.608973 using {'reg_alpha': 1.5, 'reg_lambda': 0.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/karen/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XlcV3X2+PHXAVQENwQ1FRVU1NwX\ntMyyzDKzxUpbrEYts0X7ljXZ1FST4/Qby1YrrTRNsz2nMW1zKZtKswA1VNxQMTFNwS0WRfT8/vhc\nC0kEhMv9AOf5ePDgw/vzvvdz7g07vO897/cVVcUYY4wpbQFeB2CMMaZisgRjjDHGFZZgjDHGuMIS\njDHGGFdYgjHGGOMKSzDGGGNcYQnGGGOMKyzBGGOMcYUlGGOMMa4I8joAL0VERGhUVJTXYRhjTLmS\nkJCQpqr1CutXqRNMVFQU8fHxXodhjDHliohsK0o/u0RmjDHGFZZgjDHGuMISjDHGGFdU6nswxhhv\nHTlyhNTUVA4dOuR1KOYkgoODiYyMpEqVKqe1vSUYY4xnUlNTqVmzJlFRUYiI1+GYPFSV9PR0UlNT\niY6OPq192CUyY4xnDh06RHh4uCUXPyQihIeHl2h0aQnGGOMpSy7+q6T/bSzBGFOKsnJy+Wz1TrJz\njnodijGeczXBiEh/EdkgIski8lABfa4TkSQRWSsi7+RpHyYim5yvYXnarxeRRKf/xDztTUVkiYis\ndN4f4OaxGZNXTu4xZi1LoffErxn19grufW8lR4+p12EZ4ynXEoyIBAKTgUuBtsAQEWmbr08M8DDQ\nS1XbAWOc9rrA48BZQA/gcREJE5Fw4Gmgr9O/gYj0dXb3KPCBqnYBbgCmuHVsxhx39Jjy0YpU+j73\nNY/PW0uLeqHc3rs5C5N+5cnP13kdnvEzX3/9NZdffnmJ+5zK+vXr6dmzJ9WqVeOZZ54psN/w4cOJ\njo6mc+fOdO7cmVWrVp32ZxbEzSqyHkCyqm4BEJH3gIFAUp4+I4HJqroPQFV3O+2XAItUda+z7SKg\nP5AMbFTVPU6/xcAg4EtAgVpOe23gF5eOyxhUlcXrdvPMgg1s+PU32jWqxaxbO9A7JgIR4fCRo0z7\nditREaHcdFYzr8M1RaSqqCoBAeX37kHdunV58cUXmTt3bqF9n376aQYPHuxaLG4mmMbA9jw/p+Ib\nkeTVCkBElgKBwDhV/aKAbRsDXwBtRCTKabsKqOr0GQcsFJH/A0KBi0rvUIz5w/It6Uz8Yj0rft5P\ndEQoL9/YhQHtGxIQ8McN0ccub8vPe7P4x8driQwL4fxWha4LWOn9c/5akn45WKr7bNuoFo9f0e6U\nfVJSUrj00kvp06cP33//PWPGjOHVV1/l8OHDtGjRgjfeeIMaNWrw2Wefcf/99xMREUHXrl3ZsmUL\nn3zyyUn3+eOPPzJmzBiys7OpXr06b7zxBq1btz6hz7hx49i8eTM7duxg+/btPPjgg4wcORKAjIwM\nBg8ezJo1a+jWrRtvvfUWIsL48eOZP38+2dnZnHPOObz22mt/uhFfv3596tevz6efflqCM1c63EzT\nJys/yH9ROgiIAS4AhgCvi0idgrZ1Rjp3Ae8D3wIpQK7z/hBgpqpGAgOA2SLyp+MTkdtFJF5E4vfs\n2ZP/bWMKtGbHAYbN+JEbpi5nx/5sJlzTgYX39ebyjo1OSC4AQYEBvHRjV1o1qMnot1ewYddvHkVt\nimLDhg0MHTqURYsWMX36dBYvXsyKFSuIjY3lueee49ChQ9xxxx18/vnnfPfddxT2/442bdrwzTff\nsHLlSsaPH8/f//73k/ZLTEzk008/5fvvv2f8+PH88ovvwsvKlSt54YUXSEpKYsuWLSxduhSAu+++\nm7i4ONasWUN2dvbvCe7VV1/l1VdfLfZxP/LII3Ts2JH77ruPw4cPF3v7wrg5gkkFmuT5OZI/X7ZK\nBZar6hFgq4hswJdwUvElnbzbfg2gqvOB+eBLFsDxcp0R+C6joarfi0gwEAHszrMfVHUqMBUgNjbW\n7sKaQm1Ny+TZhRv4JHEndUKq8PcBbRjaM4rgKoGn3K5GtSBmDI9l4MtLuXVmHP8dfQ71awaXUdTl\nT2EjDTc1a9aMs88+m08++YSkpCR69eoFQE5ODj179mT9+vU0b9789wmHQ4YMYerUqQXu78CBAwwb\nNoxNmzYhIhw5cuSk/QYOHEj16tWpXr06ffr04ccff6ROnTr06NGDyMhIADp37kxKSgrnnnsuS5Ys\nYeLEiWRlZbF3717atWvHFVdcwZ133lnsY54wYQJnnHEGOTk53H777Tz11FP84x//KPZ+TsXNEUwc\nECMi0SJSFd+N93n5+swF+gCISAS+S2ZbgAVAP+fGfhjQz2lDROo738OAUcDrzr5+Bvo6750JBAM2\nRDGnbdeBQzz80Woueu5/fLV+N/93YUu+ebAPt/duUWhyOa5h7epMH9advZk5jJwVb+XLfio0NBTw\n3YO5+OKLWbVqFatWrSIpKYnp06ejWry/RR977DH69OnDmjVrmD9/foGTFfNf3jr+c7Vq1X5vCwwM\nJDc3l0OHDjFq1CjmzJnD6tWrGTlyZIkmQTZs2BARoVq1atxyyy38+OOPp72vgriWYFQ1F7gbX2JY\nh6/Ca62IjBeRK51uC4B0EUkClgBjVTXdubn/L3xJKg4Yf/yGPzDJ6b8UeFJVNzrtfwVGishPwLvA\ncC3ub4UxwL7MHCZ8to7zn17CnITt/OXsZvxvbB/+2q81tYKLvyZTh8jaTLqhM4k7DnD/B6s4ZuXL\nfuvss89m6dKlJCcnA5CVlcXGjRtp06YNW7ZsISUlBYD333//lPs5cOAAjRs3BmDmzJkF9vv44485\ndOgQ6enpfP3113Tv3r3AvseTSUREBBkZGcyZM6cYR/ZnO3fuBHxJde7cubRv375E+zsZV9ciU9XP\ngM/ytf0jz2sF7ne+8m87A5hxkvYhBXxWEtCrhCGbSizzcC5vLN3Ka//bQkZOLld3acx9F7WiSd2Q\nEu+7X7szeGTAmTzx6TqeWrCehy89sxQiNqWtXr16zJw5kyFDhvx+T+KJJ56gVatWTJkyhf79+xMR\nEUGPHj1OuZ8HH3yQYcOG8dxzz3HhhRcW2K9Hjx5cdtll/Pzzzzz22GM0atSIjRs3nrRvnTp1GDly\nJB06dCAqKuqEZHT8/sudd97Jrl27iI2N5eDBgwQEBPx+L6dWrVoMGDCA119/nUaNGnHTTTexZ88e\nVJXOnTuf1j2cwkhl/iM/NjZW7YmWJif3GO/++DMvfbWJtIwcLm7bgAf6tab1GTVL9XNUlcc+XsNb\ny39mwjUdGNKjaanuvzxat24dZ55ZPpJtRkYGNWrUQFUZPXo0MTEx3Hfffae9v3HjxlGjRg0eeOCB\nUoyy9J3sv5GIJKhqbGHb2mrKptI6ekz5eNUOnlu0kdR92ZwVXZepQ9vQtWmYK58nIoy7oh3b92bz\n6Nw1NAkL4dyYCFc+y5S+adOmMWvWLHJycujSpQt33HGH1yH5PRvB2Aim0jk+SfLpBevZ+GsG7RvX\nYuwlbX6fJOm23w4d4dpXv2fH/mw+uuscYhqU7kipPClPI5iTeeONN5g0adIJbb169WLy5MkeRVT6\nSjKCsQRjCaZSyTtJsnlEKH/t15pL25/xp3ksbtuxP5urJi+lWlAA/x3Vi3o1qxW+UQW0bt062rRp\nYysq+ylVZf369aedYMrvegjGFMOaHQcY6kyS/GX/IZ50Jkle1rFhmScXgMZ1qjN9WCxpGYcZ+WY8\nh45UzvLl4OBg0tPTi10GbNx3/IFjwcGnP3fLRjA2gqnQtuzJ4NlFG/nUmSQ56oIWRZokWVa+WLOL\nu95OYED7hrw0pIsnyc5L9shk/1bQI5PtJr+p1HYeyObFLzfxQXwq1YICuOfCltzWu/lpzWNxU//2\nZ/DwpW3492fraRYewoP923gdUpmqUqXKaT+O1/g/SzCmQtmXmcMr/9vMrGUpHFPlL2c3Y3Sfln59\nj2Pkec3ZmpbFlK83ExURynWxTQrfyJhywBKMqRAyD+cy47utTP2m9CdJuk1EGD+wHan7svj7R6uJ\nrFOdc1pa+bIp/+wejN2DKdcO5x7l3R9+5uUlyaRl5NCvbQMeuKQ1rcph6e/BQ0cY/Moydh04xEej\netGyfg2vQzLmpKyKzFRoR48p/0lIpe+z/2Pc/CRa1q/BR6POYerQ2HKZXABqBVdh+rDuVA0K4JaZ\nP5KeUfrLpxtTlizBmHJFVVm4dheXTvqGv374E3VCqvDmrT14d+TZrs3AL0tN6oYwbWgsuw8e5vbZ\nCZW2fNlUDJZgTLnx/eZ0rnllGbfPTiD3qDL5xq7MG30uvVvVq1AT9bo0DeP56zuTsG0fY+ck2urL\nptyym/zG763ZcYCJCzbwzcY9nFErmCev6cDgbpEEBVbcv48GdGjI3/q34akv1hMVHsJf+7UufCNj\n/IwlGOO38k+SfGTAmfylZzO/mSTptjvPb05KWiYvfZVMs/BQBneL9DokY4rFEozxO+VlkqTbRIQn\nrm5P6v4sHv4okcZ1qtOzRbjXYRlTZFambGXKfuP4JMmZy1JQVW46qxl3X9iSiBr+O0myLBzIPsKg\nV5ax57fDfDTqHFrUs/Jl4y1bTbkILMH4h7yTJDNzcrm6SyRjLoopF5Mky8r2vVlcNXkpNYKD+O+o\nXtQNrep1SKYSs7XIjN+rSJMk3dakbghTh8YyZNpy7pgdz1u3nUW1oMpxL8qUX5ZgTJk7ekyZu9L3\nJMkd+7M5u7m7T5KsKLo1C+PZazvxf++u5ME5ibxwfecKVZ5tKh5LMKbMqCqLkn7lmYUb2PhrBh0a\n12bCNR04r4yeJFkRXNGpEdvSM3lm4UaiwkO57+JWXodkTIEswZgy8f3mdCYuWM9K50mSU27qyqXt\nz7DEchpG92nJ1rQsJn25iaiIEK7uYuXLxj9ZgjGuWp16gIkL1vPtpjQa1g7mqUEdGNS1Yk+SdJuI\nMOGaDuzYn8Xf5qymcZ0QekTX9TosY/7EqsisiswVm/dk8NzCjXy6eidhIVUYdUHLSjVJsizsz8rh\nminL2JuVw9xRvYiKCPU6JFNJ2GrKxhM7D2Tz0H8S6ff8NyzZsJt7+sbwzYN9GNm7uSWXUlYnpCpv\n3NIdAW6dGcf+rByvQzLmBHaJzJSKfZk5TPk6mVnfbwOFoT19T5Ks7JMk3dYsPJSpQ2O5adoP3DE7\ngdkjzqJqkP3daPyDJRhTIpmHc5n+3Vam2SRJz3SPqsvT13bk3vdW8dBHiTx7bScrnjB+wRKMOS2H\nc4/yzg8/M9mZJHlJuwY80K81MTZJ0hMDOzcmJS2L5xdvJDo8lP/rG+N1SMZYgjHFk3+SZM/m4Uwb\n2pouNknSc/f0bcm29EyeXbSRpuEhDOzc2OuQTCVnCcYUiaqyMOlXns0zSfLJQR04t6VNkvQXIsKE\nQR1I3Z/N2DmJRIZVp1szK1823rG7gaZQyzancfWUZdwxO4HcY8qUm7oy7+5enBdTsZ4kWRFUCwrk\ntZu70bhOdUa+mcC29EyvQzKVmCUYU6DVqQf4y/QfuHHaD/x68BBPDerAwjG9GdChoSUWPxYWWpUZ\nw7tzTJVbZsZxIOuI1yGZSsoSjPmTzXsyGP32Cq54+TvW7DjAo5edyZIHLuD67k1tBn45ER0Ryms3\nd2P73izufCuBnNxjXodkKiG7B2N+t/NANpMWb+LDhFSCgwK4p28MI8+LpmYle5JkRXFW83AmDu7I\nfe//xCP/Xc3EwR1t5GnKlCUYY5MkK7Cru0SyNS2LF7/cRFREKKP7tPQ6JFOJWIKpxPJPkrymq2+S\nZGSYTZKsSO67KIZt6Zk8vWADzcJDuLxjI69DMpWEJZhK6PgkyZe/SiY90yZJVnQiwlODOrJjXzb3\nf/ATjepUt4e7mTJhqylXotWUjx5T/rtyB887kyTPaRHO2EtskmRlsTczh6unLCXjUC5zR/ey5XzM\nafOL1ZRFpL+IbBCRZBF5qIA+14lIkoisFZF38rQPE5FNztewPO3Xi0ii039iUfZV2akqC9buov8L\n3/DAhz9RN7Qqb404i3dGnm3JpRKp65Qv5x5zypezrXzZuMu1EYyIBAIbgYuBVCAOGKKqSXn6xAAf\nABeq6j4Rqa+qu0WkLhAPxAIKJADd8CXElUA3Vd0jIrOAN1X1y4L2daoYK8MIZtnmNCZ+sYFV2/fT\nvF4oY/u1pr89SbJS+35zOkNn/ECP6LrMvKUHVaz03BSTP4xgegDJqrpFVXOA94CB+fqMBCar6j6A\nPAnhEmCRqu513lsE9AeaAxtVdY/TbzEwqJB9VUr5J0lOHNSRhWN6c6lNkqz0erYIZ8I1HVmanM5j\nc9dQmS+TG3e5eZO/MbA9z8+pwFn5+rQCEJGlQCAwTlW/KGDbxsAXQBsRiXLargKqFrKvSmXzngye\nXbiBz1bvIiykCo9ediY3n21PkjQnGtwtkpS0TF5ekkxURCh3nt/C65BMBeRmgjnZn8n5/1QKAmKA\nC4BI4FsRaV/Qts6lr7uA94FjwDJ8o5oC96Wq+08ISuR24HaApk2bFv+o/NQv+32TJOes8E2SvLdv\nDLfZJElzCvdf3IqU9Eye/Hw9zeqGcGmHhl6HZCoYNxNMKtAkz8+RwC8n6bNcVY8AW0VkA74kkYov\nUeTd9msAVZ0PzIffk8XRQvYVl/cDVXUqMBV892BKdIR+YG9mDlOWJPPmct8kyWE9oxjdpwXhNknS\nFCIgQHjm2k78sj+bMe+vomGd6nRuUsfrsEwF4uY9mDggRkSiRaQqcAMwL1+fuUAfABGJwHeZawuw\nAOgnImEiEgb0c9oQkfrO9zBgFPB6IfuqkDIO5zJp8SZ6T1zCjKVbGdipEV89cD7/uKKtJRdTZMFV\nApk2NJb6tapx26x4UvdleR2SqUBcG8Goaq6I3I0vMQQCM1R1rYiMB+JVdR5/JJIkfCORsaqaDiAi\n/+KP0cd4Vd3rvJ4kIp3ytG90Xhe4r4rkcO5R3l7ue5JkemYO/dudwV/7tbJJkua0hdeoxhvDu3P1\nlGXcOjOOOXedQy27tGpKgU20LCdlykePKR+tSOWFxZvYsT+bXi3DGXtJG7ukYUrNsuQ0hs74kZ4t\nwpkxvLuVL5sC+UOZsikFqsoXa3yTJMfOSSS8hm+S5Nu3nW3JxZSqc1pG8O+rO/DtpjQen7fWypdN\nidlaZH5sWXIaTy3YwE/b99OiXiiv3NTVJkkaV13XvQlb0zN55evNRIeHMrJ388I3MqYAlmD8UGLq\nfp5esIFvN6XRqHYwEwd15Jquje1hX6ZMjO3Xmp/Ts/j35+toGh7CJe3O8DokU05ZgvEjybszeG6R\nb5Jk3dCqNknSeCIgQHj2uk7s2J/Nve+t5IM7etIx0i7HmuKzm/x+cJP/+CTJDxO2U71KILed19wm\nSRrP7fntMFdPWcrh3GPMHd2LxnWqex2S8RNFvclvIxgP5Z8kOfycaJskafxGvZq+8uVrpixjxMw4\nPryzp/3RY4rFEowHMg7nMv3brUz7dgtZObkM6hrJmItb2V+Ixu/ENKjJlJu7MvyNOO5+ZyXTh8Xa\nvUBTZJZgytDJJkk+cEkrWta3SZLGf50XU48nrmrPwx+t5p/zkxg/sJ1VMpoisQRTBmySpCnvhvRo\nSkpaJq99s4WoiFBGnBvtdUimHLAE4yLfkyR/5ZmFG0jenUHHyNo8Nagj58ZEeB2aMcX2t/5t2Jae\nxROfJtG0bggXt23gdUjGz9nFVJcsS07jqinLuPOtBFSVV2/uyseje1lyMeVWQIDw/PWd6di4Nve8\nu5I1Ow54HZLxc5ZgSlli6n5ufv0Hbnz9B/YcPMTEwR1ZMKY3/dvbkyRN+Ve9aiDThsVSN7QqI2bF\nsfNAttchGT9mCaaUJO/O4K63Erjy5aUk7TzIY5e35asHLuC62CZWdWMqlPo1g5kxvDuZh49y68x4\nMg7neh2S8VN2D6aEduzPZtLijcxJSKV6lUDGXBTDiHNtkqSp2FqfUZPJN3Xl1plx3PPuSqb+pZv9\nIWX+xBLMadqbmcPkJcnMdiZJ3tIrmlEX2CRJU3mc36oe/7yyHY/OXcMTn65j3JXtvA7J+BlLMKdh\nTkIq4+atJSsnl8HdIrn3IpskaSqnm89uRkpaJq9/t5Wo8BCG97LyZfMHSzCnoVHtYM6LieCv/WyS\npDEPDziTbXuzGP9JEk3DQ7iwjZUvGx9b7NIPFrs0przLysnl+teWs3lPBh/e2ZN2jWp7HZJxkT3R\n0hhTZkKqBvH6sFhqV6/CiJnx7DpwyOuQjB+wBGOMKRUNavnKl387dIQRs+LItPLlSs8SjDGm1JzZ\nsBYv39iVdTsPcu97Kzl6rPJegjfFTDAiEiAitdwKxhhT/vVpU59xV7Zj8brd/L9P13kdjvFQoQlG\nRN4RkVoiEgokARtEZKz7oRljyquhPaO4pVcUM5ZuZfb3KV6HYzxSlBFMW1U9CFwFfAY0Bf7ialTG\nmHLv0cvactGZ9Xl83lqWbNjtdTjGA0VJMFVEpAq+BPOxqh4B7MKqMeaUAgOESTd04cyGtbj77RWs\n23nQ65BMGStKgnkNSAFCgW9EpBlgvynGmEKFVgti+rDu1AyuwoiZcew+aOXLlUmhCUZVX1TVxqo6\nQH22AX3KIDZjTAVwRu1gpg+PZX/2EUbMiicrx8qXK4ui3OS/17nJLyIyXURWABeWQWzGmAqiXaPa\nvDSkC2t/OcCY91ZZ+XIlUZRLZLc6N/n7AfWAW4AnXY3KGFPh9D2zAY9d3paFSb/y5OdWvlwZFGWx\ny+OPYRwAvKGqP4k9mtEYcxpu6RVNSlom077dSlREKDed1czrkIyLipJgEkRkIRANPCwiNYFj7oZl\njKmoHru8LT/vzeIfH68lMiyE81vV8zok45KiXCIbATwEdFfVLKAqvstkxhhTbEGBAbx0Y1daNajJ\n6LdXsGHXb16HZFxSlCqyY0Ak8KiIPAOco6qJrkdmjKmwalQLYsbwWEKqBnLrzDh2/2blyxVRUarI\nngTuxbdMTBJwj4hMcDswY0zF1rB2daYP687ezBxGzoonO+eo1yGZUlaUS2QDgItVdYaqzgD6A5e5\nG5YxpjLoEFmbSTd0JnHHAe7/YBXHrHy5Qinqasp18ry2R9UZY0pNv3Zn8MiAM/l8zS6eWrDe63BM\nKSpKFdkEYKWILMFXstwbeNjVqIwxlcqIc6NJSc/ktf9tISo8lCE9mnodkikFhSYYVX1XRL4GuuNL\nMH/DHlRmjClFIsK4K9qxfW82j85dQ5OwEM6NifA6LFNCRUoUqrpTVeep6sequgtYXpTtRKS/iGwQ\nkWQReaiAPteJSJKIrBWRd/K0DxORTc7XsDzt14tIotN/4kn2N1hEVERiixKjMcY/BAUG8PKNXYip\nX4O73k5g069Wvlzene5IpNCZ/CISCEwGLgXaAkNEpG2+PjH4Lrf1UtV2wBinvS7wOHAW0AN4XETC\nRCQceBro6/RvICJ98+yvJnAP8MNpHpcxxkM1g6swfXh3qgUFcsvMOPb8dtjrkEwJnG6CKUqpRw8g\nWVW3qGoO8B4wMF+fkcBkVd0HoKrHn0p0CbBIVfc67y3CV73WHNioqnucfouBQXn29y9gImBF9caU\nU43rVGf6sFjSMg4z8s14Dh2x8uXyqsB7MCLyEidPJMKJVWUFaQxsz/NzKr4RSV6tnM9aCgQC41T1\niwK2bQx8AbQRkSin7Sp8KwsgIl2AJqr6iYg8UIT4jDF+qlOTOrxwfWfuensFf/3gJ14a0oWAAFsC\nsbw51U3++NN877iT/TbkT1hBQAxwAb7VAr4VkfYFbauq+0TkLuB9fOuhLQOai0gA8DwwvNCgRG4H\nbgdo2tQqVYzxV/3bN+Sh/m2Y8Pl6oiJCGHtJG69DMsVUYIJR1Vkl3Hcq0CTPz5HALyfps9x5DPNW\nEdmAL+Gk4ks6ebf92olrPjAffk8WR4GaQHvga2eh5zOAeSJypaqekAxVdSowFSA2NtZmdRnjx27v\n3ZyU9EwmL9lMs/BQrottUvhGxm+4WW4cB8SISLSIVAVuAObl6zMX5+mYIhKB75LZFmAB0M+5sR+G\n71k0C5x+9Z3vYcAo4HVVPaCqEaoapapR+Krc/pRcjDHli4gwfmB7zouJ4O8frWbZ5jSvQzLF4FqC\nUdVc4G58iWEd8IGqrhWR8SJypdNtAZAuIknAEmCsqqar6l58N+zjnK/xThvAJKf/UuBJVd3o1jEY\nY7xXJTCAyTd1JToilDtnJ5C8O8PrkEwRiWrlvUoUGxur8fE2yDGmPNi+N4urpywlpGoQ/x11DuE1\nqnkdUqUlIgmqWuhcw0Jn8ovIiydpPgDEq+rHpxOcMcYUV5O6IUwbGssNU5dz++wE3r7tLIKrBHod\nljmFolwiCwY6A5ucr45AXWCEiLzgYmzGGHOCLk3DeP76ziRs28eDcxKpzFdgyoOiLHbZErjQuaeC\niLwCLAQuBla7GJsxxvzJgA4NebB/ayZ+sYGo8BDu79fa65BMAYqSYBoDofgui+G8bqSqR0XE1nEw\nxpS5u85vwba0LF78Kplm4aEM6hbpdUjmJIqSYCYCq5wVlY8v1/9vEQnFt1SLMcaUKRHhiavbs31f\nFg99lEjjsOqc3Tzc67BMPkWqIhORhvjWFhPgR1XNP2GyXLIqMmPKtwNZR7jmlaWkZeTw31Hn0Lxe\nDa9DqhSKWkVW1Hkw3YHzgHOBbiUJzBhjSkvtkCq8MbwHQQHCrTPj2JuZ43VIJo9CE4yIPAncCyQ5\nX/eIyAS3AzPGmKJoGh7C1KGx/HLgEHfMjudwrq2+7C+KMoIZAFysqjNUdQa+ZfMvczcsY4wpum7N\nwnj22k7Epezjof+stvJlP1GUm/zgW57/+FIttV2KxRhjTtsVnRqxLT2TZxZupFl4CGMuauV1SJVe\nURLMBGCliCzhjyqyh12NyhhjTsPoPi3ZmpbFC4s3ERUeylVdGnsdUqVWaIJR1XedEuXu+BLM33B3\nFWZjjDktIsKEazqwY38WD85JpFGd6vSIrut1WJVWkRKFqu5U1Xmq+rGq7sK3HL4xxvidqkEBvHpz\nNyLDqnPH7HhS0jK9DqnSOt031/9jAAATaklEQVSRiD271Bjjt+qEVOWNW7oDcOvMOPZnWfmyF043\nwViJhjHGrzULD2Xq0FhS92Vzx+wEcnKPeR1SpVPgPRgReYmTJxLBV1VmjDF+rXtUXZ6+tiP3vreK\nhz5K5NlrO+E8Vt2UgVPd5D/VGiq2vooxplwY2LkxKWlZPL94I9Hhofxf3xivQ6o0Ckwwqjorf5uI\nnOHc5DfGmHLjnr4t2ZaeybOLNtI0PISBna18uSwU9x7MZ65EYYwxLhIRJgzqQI/ouoydk0jCtr2F\nb2RKrLgJxi5eGmPKpWpBgbx2czca16nOyDcT2JZu5ctuK26CmeZKFMYYUwbCQqsyY3h3jqlyy8w4\nDmQd8TqkCq1YCUZVp7gViDHGlIXoiFBeu7kb2/dmcedbVr7sJlvyxRhT6ZzVPJyJgzvy/ZZ0Hvmv\nrb7slqKupmyMMRXK1V0i2ZqWxYtfbiIqIpTRfVp6HVKFYwnGGFNp3XdRDNvSM3l6wQaahYdwecdG\nXodUodglMmNMpSUiPDWoI7HNwrj/g59Y8fM+r0OqUCzBGGMqteAqgUwdGkvD2sGMnBXP9r1ZXodU\nYViCMcZUenWd8uXcY075craVL5cGSzDGGAO0qFeDV2/uxrb0TEa9ncCRo1a+XFKWYIwxxtGzRTgT\nrunI0uR0Hpu7xsqXS8iqyIwxJo/B3SJJScvk5SXJREWEcuf5LbwOqdyyBGOMMfncf3ErUtIzefLz\n9TSrG8KlHRp6HVK5ZJfIjDEmn4AA4ZlrO9G1aR3GvL+KVdv3ex1SuWQJxhhjTiK4SiDThsZSv1Y1\nbpsVT+o+K18uLkswxhhTgPAa1XhjeHcO5x7l1plxHDxk5cvFYQnGGGNOoWX9mrx2cze27Mlk9Nsr\nrHy5GCzBGGNMIc5pGcG/r+7At5vSeHzeWitfLiKrIjPGmCK4rnsTtqZn8srXm4kOD2Vk7+Zeh+T3\nXB3BiEh/EdkgIski8lABfa4TkSQRWSsi7+RpHyYim5yvYXnarxeRRKf/xDzt9zv7SRSRL0WkmZvH\nZoypfMb2a81lHRry78/XsWDtLq/D8XuuJRgRCQQmA5cCbYEhItI2X58Y4GGgl6q2A8Y47XWBx4Gz\ngB7A4yISJiLhwNNAX6d/AxHp6+xuJRCrqh2BOcBEjDGmFAUECM9e14lOkXW4972VJKZa+fKpuDmC\n6QEkq+oWVc0B3gMG5uszEpisqvsAVHW3034JsEhV9zrvLQL6A82Bjaq6x+m3GBjkbLtEVY/XES4H\nIl06LmNMJXa8fDmiRjVGzIpnx/5sr0PyW24mmMbA9jw/pzptebUCWonIUhFZLiL9C9k2GWgjIlEi\nEgRcBTQ5yWePAD4vhWMwxpg/qVfTV758KOcoI2bG8ZuVL5+UmwlGTtKWv/QiCIgBLgCGAK+LSJ2C\ntnVGM3cB7wPfAilA7gkfKnIzEIvvUtqfgxK5XUTiRSR+z549J+tijDGFimlQkyk3d2XT7gzufmcl\nuVa+/CduJphUThxdRAK/nKTPx6p6RFW3AhvwJZwCt1XV+ap6lqr2dPpvOt5JRC4CHgGuVNXDJwtK\nVaeqaqyqxtarV69EB2iMqdzOi6nHE1e1538b9/DP+UlWvpyPmwkmDogRkWgRqQrcAMzL12cu0AdA\nRCLwXTLbAiwA+jk39sOAfk4bIlLf+R4GjAJed37uAryGL7nsxhhjysCQHk25o3dzZi/fxoylKV6H\n41dcmwejqrkicje+xBAIzFDVtSIyHohX1Xn8kUiSgKPAWFVNBxCRf+FLUgDjVXWv83qSiHTK077R\nef00UAP4UEQAflbVK906PmOMOe5v/duwLT2LJz5NomndEC5u28DrkPyCVOYhXWxsrMbHx3sdhjGm\nAsjOOcoNU79n468ZfHhnT9o3ru11SK4RkQRVjS2sny0VY4wxpaB61UCmDYulbmhVRsyKY+cBK1+2\nBGOMMaWkfs1gZgzvTubho9w6M56Mw7mFb1SBWYIxxphS1PqMmky+qSsbf/2Ne96t3OXLlmCMMaaU\nnd+qHv+8sh1frd/NE5+u8zocz9hqysYY44Kbz25GSlomr3+3lajwEIb3ivY6pDJnCcYYY1zy8IAz\n2bY3i/GfJNE0PIQL21Su8mW7RGaMMS4JDBAm3dCZdo1qc/c7K1n7ywGvQypTlmCMMcZFIVWDeH1Y\nLLWrV2HEzHh2HTjkdUhlxhKMMca4rEEtX/nyb4eOMGJWHJmVpHzZEowxxpSBMxvW4uUbu7Ju50Hu\nfW8lR49V/FVULMEYY0wZ6dOmPuOubMfidbv5f5WgfNmqyIwxpgwN7RnF1rRMZizdSnRECH/pGeV1\nSK6xBGOMMWXs0cva8nN6Fo/PW0tk3RD6tK7vdUiusEtkxhhTxgIDhBeHdKHNGbW4++0VrNt50OuQ\nXGEJxhhjPBBaLYjpw2OpERzEiJlx7D5Y8cqXLcEYY4xHGtauzvRh3dmffYTb3ownK6dilS9bgjHG\nGA+1b1ybF2/owpodB7jv/VUcq0Dly5ZgjDHGYxe1bcCjl7VlwdpfefKL9V6HU2qsiswYY/zALb2i\nSEnPZOo3W4gKD+XGs5p6HVKJWYIxxhg/ICL84/K2/Lw3i8c+XkNkWHV6t6rndVglYpfIjDHGTwQF\nBvDyjV2JqV+D0W+vYMOu37wOqUQswRhjjB+pUS2IGcO7U71qILfOjGPPb4e9Dum0WYIxxhg/06iO\nr3x5b2YOt70ZT3bOUa9DOi2WYIwxxg91iKzNpBs6k5i6n/s/KJ/ly5ZgjDHGT/VrdwaPDDiTz9fs\nYuKCDV6HU2xWRWaMMX5sxLnRbE3L5NX/bSYqPIQbepSf8mVLMMYY48dEhH9e2Y7t+7J5dO4amtQN\noVfLCK/DKhK7RGaMMX4uKDCAyTd2oUW9Gtz5VgKbfi0f5cuWYIwxphyoGVyF6cNjqRYUyC0z40jL\n8P/yZUswxhhTTkSGhTB9WCxpGYcZ+WY8h474d/myJRhjjClHOjWpwwvXd2bV9v389cOf/Lp82RKM\nMcaUM/3bN+Sh/m34NHEnzy7y3/JlqyIzxphy6PbezUlJz2Tyks00Cw/lutgmXof0J5ZgjDGmHBIR\nxg9sT+q+bP7+0Woiw6pzTgv/Kl+2S2TGGFNOVQkMYPJNXYmOCOXO2Qkk787wOqQTWIIxxphyrFZw\nFWYM707VoABunRlHuh+VL1uCMcaYcq5J3RCmDY3l14OHuH12gt+UL1uCMcaYCqBL0zCev74zCdv2\n8eCcRFS9L192NcGISH8R2SAiySLyUAF9rhORJBFZKyLv5GkfJiKbnK9hedqvF5FEp//EPO3VROR9\n57N+EJEoN4/NGGP8zYAODXmwf2vm/fQLzy/a6HU47lWRiUggMBm4GEgF4kRknqom5ekTAzwM9FLV\nfSJS32mvCzwOxAIKJIjIPHwJ8Wmgm6ruEZFZItJXVb8ERgD7VLWliNwAPAVc79bxGWOMP7rr/BZs\nS8vixa+SaRYeyqBukZ7F4uYIpgeQrKpbVDUHeA8YmK/PSGCyqu4DUNXdTvslwCJV3eu8twjoDzQH\nNqrqHqffYmCQ83ogMMt5PQfoKyLiwnEZY4zfEhGeuLo957QI56GPElm+Jd2zWNxMMI2B7Xl+TnXa\n8moFtBKRpSKyXET6F7JtMtBGRKJEJAi4CmiSfxtVzQUOAOGleDzGGFMuVAkM4JWbutG0bgh3zE5g\nyx5vypfdTDAnGz3kv+sUBMQAFwBDgNdFpE5B2zqjmbuA94FvgRQgtxifh4jcLiLxIhK/Z8+ek2xi\njDHlX+2QKrwxvAdBAcKtM+PYm5lT5jG4mWBS+WN0ARAJ/HKSPh+r6hFV3QpswJdwCtxWVeer6lmq\n2tPpvyn/5zmjm9rA3vxBqepUVY1V1dh69eqV8BCNMcZ/NQ0PYerQWH45cIg7ZsdzOLdsy5fdTDBx\nQIyIRItIVeAGYF6+PnOBPgAiEoHvktkWYAHQT0TCRCQM6Oe0kacQIAwYBbzu7GsecLzabDDwlfpD\nnZ4xxnioW7Mwnr22E3Ep+3joP6vLtHzZtSoyVc0VkbvxJYZAYIaqrhWR8UC8qs7jj0SSBBwFxqpq\nOoCI/AtfkgIYr6rHRyOTRKRTnvbjtXjTgdkikoxv5HKDW8dmjDHlyRWdGrEtPZNnFm6kWXgIYy5q\nVSafK5X5j/zY2FiNj4/3OgxjjHGdqvLAh4n8Z0UqL1zfmau65K+5KjoRSVDV2ML62WrKxhhTCYgI\nE67pwI79WTw4J5FGdarTI7quq59pS8UYY0wlUTUogFdv7kabhjXJysktfIMSshGMMcZUInVCqjJ3\nVC8CAtyfh24jGGOMqWTKIrmAJRhjjDEusQRjjDHGFZZgjDHGuMISjDHGGFdYgjHGGOMKSzDGGGNc\nYQnGGGOMKyr1WmQisgfY5nUcJRQBpHkdhB+x8/EHOxcnsvNxopKcj2aqWujzTip1gqkIRCS+KIvO\nVRZ2Pv5g5+JEdj5OVBbnwy6RGWOMcYUlGGOMMa6wBFP+TfU6AD9j5+MPdi5OZOfjRK6fD7sHY4wx\nxhU2gjHGGOMKSzB+TET6i8gGEUkWkYdO8n41EXnfef8HEYly2i8WkQQRWe18v7CsYy9tp3su8rzf\nVEQyROSBsorZTSU5HyLSUUS+F5G1zu9IcFnG7oYS/FupIiKznPOwTkQeLuvYS1sRzkVvEVkhIrki\nMjjfe8NEZJPzNazEwaiqffnhFxAIbAaaA1WBn4C2+fqMAl51Xt8AvO+87gI0cl63B3Z4fTxenYs8\n7/8H+BB4wOvj8fh3IwhIBDo5P4cDgV4fk4fn40bgPed1CJACRHl9TC6fiyigI/AmMDhPe11gi/M9\nzHkdVpJ4bATjv3oAyaq6RVVzgPeAgfn6DARmOa/nAH1FRFR1par+4rSvBYJFpFqZRO2O0z4XACJy\nFb5/LGvLKF63leR89AMSVfUnAFVNV9WjZRS3W0pyPhQIFZEgoDqQAxwsm7BdUei5UNUUVU0EjuXb\n9hJgkaruVdV9wCKgf0mCsQTjvxoD2/P8nOq0nbSPquYCB/D9RZrXIGClqh52Kc6ycNrnQkRCgb8B\n/yyDOMtKSX43WgEqIgucyyQPlkG8bivJ+ZgDZAI7gZ+BZ1R1r9sBu6go58KNbU8qqCQbG1ed7Jmm\n+Uv+TtlHRNoBT+H7q7U8K8m5+CfwvKpmOAOaiqAk5yMIOBfoDmQBX4pIgqp+WbohlqmSnI8ewFGg\nEb7LQt+KyGJV3VK6IZaZopwLN7Y9KRvB+K9UoEmenyOBXwrq4wzxawN7nZ8jgf8CQ1V1s+vRuqsk\n5+IsYKKIpABjgL+LyN1uB+yykpyPVOB/qpqmqlnAZ0BX1yN2V0nOx43AF6p6RFV3A0uB8rycTFHO\nhRvbnpQlGP8VB8SISLSIVMV3Y3Jevj7zgOOVHoOBr1RVRaQO8CnwsKouLbOI3XPa50JVz1PVKFWN\nAl4A/q2qL5dV4C457fMBLAA6ikiI8z/a84GkMorbLSU5Hz8DF4pPKHA2sL6M4nZDUc5FQRYA/UQk\nTETC8F35WFCiaLyuerCvU1aEDAA24qsKecRpGw9c6bwOxlcZlQz8CDR32h/Fd115VZ6v+l4fjxfn\nIt8+xlEBqshKej6Am/EVPKwBJnp9LF6eD6CG074WX6Id6/WxlMG56I5vtJIJpANr82x7q3OOkoFb\nShqLzeQ3xhjjCrtEZowxxhWWYIwxxrjCEowxxhhXWIIxxhjjCkswxhhjXGEJxhhjjCsswRjjEREZ\nLiKlMulTRFJEJKII/TJK4/OMKQpLMMYUkzPr2/7tGFMI+0diTBGISJTzQKopwArgL85Du1aIyIci\nUsPpN0BE1ovIdyLyooh8UsT9X+E8CGuliCwWkQZO+zjngVgLnVHKNSIy0XlA1hciUiXPbsaKyI/O\nV0tn+2gnzjgR+Veez6shIl868a8WkfzL2xtTYpZgjCm61vge0nQxMAK4SFW7AvHA/eJ7MuRrwKWq\nei5Qrxj7/g44W1W74HuGR95l9FsAl+F7rsdbwBJV7QBkO+3HHVTVHsDL+NZdA5gEvKKq3YFdefoe\nAq524u8DPHv8+TnGlBZLMMYU3TZVXY5vQcS2wFIRWYVvEcVmQBtgi6pudfq/W4x9RwILRGQ1MBZo\nl+e9z1X1CLAa3xMLv3DaV+N7OuFx7+b53tN53StP++w8fQX4t4gkAovxPfejQTHiNaZQ9jwYY4ou\n0/ku+J78NyTvmyLSpQT7fgl4TlXnicgF+BbmPO4wgKoeE5Ej+scCgsc48d+wFuH1cTfhG2F1U9Uj\nzuMMgksQvzF/YiMYY4pvOdArz32OEBFphW+Z9+YiEuX0u74Y+6wN7HBeDztVx1O4Ps/3753XS/Et\n2Q6+pJL383Y7yaUPvhGYMaXKRjDGFJOq7hGR4cC7IlLNaX5UVTeKyCjgCxFJw7csfFGNAz4UkR34\nElj0aYRWTUR+wPeH4/HR1b3AOyJyL/CfPH3fBuaLSDy+xzmU52egGD9ly/UbU4pEpIb6Hs8swGRg\nk6o+73VcxnjBLpEZU7pGOjf+1+K7DPWax/EY4xkbwRjjMhG5Bd+lqryWqupoL+IxpqxYgjHGGOMK\nu0RmjDHGFZZgjDHGuMISjDHGGFdYgjHGGOMKSzDGGGNc8f8BCDkbVXH085sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a0b876438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_2.best_score_, gsearch5_2.best_params_))\n",
    "test_means = gsearch5_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_2.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_lambda' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "经过多次微调，最佳参数组合：{'reg_alpha': 1.5, 'reg_lambda': 0.1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, colsample_bylevel=0.7, colsample_bytree=0.9,\n",
       "       gamma=0, learning_rate=0.1, max_delta_step=0, max_depth=5,\n",
       "       min_child_weight=4, missing=None, n_estimators=124, nthread=-1,\n",
       "       objective='multi:softprob', reg_alpha=1.5, reg_lambda=0.1,\n",
       "       scale_pos_weight=1, seed=3, silent=True, subsample=0.8)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_xgb = gsearch5_2.best_estimator_\n",
    "best_xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save model to file\n",
    "import pickle\n",
    "pickle.dump(best_xgb, open(\"rent.pickle.dat\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
